/*
* Conditions Of Use 
* 
* This software was developed by employees of the National Institute of
* Standards and Technology (NIST), an agency of the Federal Government.
* Pursuant to title 15 Untied States Code Section 105, works of NIST
* employees are not subject to copyright protection in the United States
* and are considered to be in the public domain.  As a result, a formal
* license is not needed to use the software.
* 
* This software is provided by NIST as a service and is expressly
* provided "AS IS."  NIST MAKES NO WARRANTY OF ANY KIND, EXPRESS, IMPLIED
* OR STATUTORY, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTY OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT
* AND DATA ACCURACY.  NIST does not warrant or make any representations
* regarding the use of the software or the results thereof, including but
* not limited to the correctness, accuracy, reliability or usefulness of
* the software.
* 
* Permission to use this software is contingent upon your acceptance
* of the terms of this agreement
*  
* .
* 
*/
package gov.nist.core;

import java.text.ParseException;
import java.util.Hashtable;

/** A lexical analyzer that is used by all parsers in our implementation.
 *
 *@version 1.2 
 *@since 1.1
 *
 *@author M. Ranganathan 
 */
public class LexerCore extends StringTokenizer {

	// IMPORTANT - All keyword matches should be between START and END
	public static final int START = 2048;
	public static final int END = START + 2048;
	// IMPORTANT -- This should be < END
	public static final int ID = END - 1;
	public static final int SAFE = END - 2;
	// Individial token classes.
	public static final int WHITESPACE = END + 1;
	public static final int DIGIT = END + 2;
	public static final int ALPHA = END + 3;
	public static final int BACKSLASH = (int) '\\';
	public static final int QUOTE = (int) '\'';
	public static final int AT = (int) '@';
	public static final int SP = (int) ' ';
	public static final int HT = (int) '\t';
	public static final int COLON = (int) ':';
	public static final int STAR = (int) '*';
	public static final int DOLLAR = (int) '$';
	public static final int PLUS = (int) '+';
	public static final int POUND = (int) '#';
	public static final int MINUS = (int) '-';
	public static final int DOUBLEQUOTE = (int) '\"';
	public static final int TILDE = (int) '~';
	public static final int BACK_QUOTE = (int) '`';
	public static final int NULL = (int) '\0';
	public static final int EQUALS = (int) '=';
	public static final int SEMICOLON = (int) ';';
	public static final int SLASH = (int) '/';
	public static final int L_SQUARE_BRACKET = (int) '[';
	public static final int R_SQUARE_BRACKET = (int) ']';
	public static final int R_CURLY = (int) '}';
	public static final int L_CURLY = (int) '{';
	public static final int HAT = (int) '^';
	public static final int BAR = (int) '|';
	public static final int DOT = (int) '.';
	public static final int EXCLAMATION = (int) '!';
	public static final int LPAREN = (int) '(';
	public static final int RPAREN = (int) ')';
	public static final int GREATER_THAN = (int) '>';
	public static final int LESS_THAN = (int) '<';
	public static final int PERCENT = (int) '%';
	public static final int QUESTION = (int) '?';
	public static final int AND = (int) '&';
	public static final int UNDERSCORE = (int) '_';

	protected static final Hashtable globalSymbolTable;
	protected static final Hashtable lexerTables;
	protected Hashtable currentLexer;
	protected String currentLexerName;
	protected Token currentMatch;

	static {
		globalSymbolTable = new Hashtable();
		lexerTables = new Hashtable();
	}

	protected void addKeyword(String name, int value) {
		// System.out.println("addKeyword " + name + " value = " + value);
		// new Exception().printStackTrace();
		Integer val = new Integer(value);
		currentLexer.put(name, val);
		if (!globalSymbolTable.containsKey(val))
			globalSymbolTable.put(val, name);
	}

	public String lookupToken(int value) {
		if (value > START) {
			return (String) globalSymbolTable.get(new Integer(value));
		} else {
			Character ch = new Character((char) value);
			return ch.toString();
		}
	}

	protected Hashtable addLexer(String lexerName) {
		currentLexer = (Hashtable) lexerTables.get(lexerName);
		if (currentLexer == null) {
			currentLexer = new Hashtable();
			lexerTables.put(lexerName, currentLexer);
		}
		return currentLexer;
	}

	//public abstract void selectLexer(String lexerName);

	public void selectLexer(String lexerName) {
		this.currentLexerName = lexerName;
	}

	protected LexerCore() {
		this.currentLexer = new Hashtable();
		this.currentLexerName = "charLexer";
	}

	/** Initialize the lexer with a buffer.
	 */
	public LexerCore(String lexerName, String buffer) {
		super(buffer);
		this.currentLexerName = lexerName;
	}

	/** Peek the next id but dont move the buffer pointer forward.
	 */

	public String peekNextId() {
		int oldPtr = ptr;
		String retval = ttoken();
		savedPtr = ptr;
		ptr = oldPtr;
		return retval;
	}

	/** Get the next id.
	 */
	public String getNextId() {
		return ttoken();
	}

	// call this after you call match
	public Token getNextToken() {
		return this.currentMatch;

	}

	/** Look ahead for one token.
	 */
	public Token peekNextToken() throws ParseException {
		return (Token) peekNextToken(1)[0];
	}

	public Token[] peekNextToken(int ntokens) throws ParseException {
		int old = ptr;
		Token[] retval = new Token[ntokens];
		for (int i = 0; i < ntokens; i++) {
			Token tok = new Token();
			if (startsId()) {
				String id = ttoken();
				tok.tokenValue = id;
                String idUppercase = id.toUpperCase();
                if (currentLexer.containsKey(idUppercase)) {
                    Integer type = (Integer) currentLexer.get(idUppercase);
                    tok.tokenType = type.intValue();
                } else
                    tok.tokenType = ID;
			} else {
				char nextChar = getNextChar();
				tok.tokenValue = String.valueOf(nextChar);
				if (isAlpha(nextChar)) {
					tok.tokenType = ALPHA;
				} else if (isDigit(nextChar)) {
					tok.tokenType = DIGIT;
				} else
					tok.tokenType = (int) nextChar;
			}
			retval[i] = tok;
		}
		savedPtr = ptr;
		ptr = old;
		return retval;
	}

	/** Match the given token or throw an exception if no such token
	 * can be matched.
	 */
	public Token match(int tok) throws ParseException {
		if (Debug.parserDebug) {
			Debug.println("match " + tok);
		}
		if (tok > START && tok < END) {
			if (tok == ID) {
				// Generic ID sought.
				if (!startsId())
					throw new ParseException(buffer + "\nID expected", ptr);
				String id = getNextId();
				this.currentMatch = new Token();
				this.currentMatch.tokenValue = id;
				this.currentMatch.tokenType = ID;
			} else if (tok == SAFE) {
				if (!startsSafeToken())
					throw new ParseException(buffer + "\nID expected", ptr);
				String id = ttokenSafe();
				this.currentMatch = new Token();
				this.currentMatch.tokenValue = id;
				this.currentMatch.tokenType = SAFE;
			} else {
				String nexttok = getNextId();
				Integer cur = (Integer) currentLexer.get(nexttok.toUpperCase());

				if (cur == null || cur.intValue() != tok)
					throw new ParseException(
						buffer + "\nUnexpected Token : " + nexttok,
						ptr);
				this.currentMatch = new Token();
				this.currentMatch.tokenValue = nexttok;
				this.currentMatch.tokenType = tok;
			}
		} else if (tok > END) {
			// Character classes.
			char next = lookAhead(0);
			if (tok == DIGIT) {
				if (!isDigit(next))
					throw new ParseException(buffer + "\nExpecting DIGIT", ptr);
				this.currentMatch = new Token();
				this.currentMatch.tokenValue =
					String.valueOf(next);
				this.currentMatch.tokenType = tok;
				consume(1);

			} else if (tok == ALPHA) {
				if (!isAlpha(next))
					throw new ParseException(buffer + "\nExpecting ALPHA", ptr);
				this.currentMatch = new Token();
				this.currentMatch.tokenValue =
					String.valueOf(next);
				this.currentMatch.tokenType = tok;
				consume(1);

			}

		} else {
			// This is a direct character spec.
			char ch = (char) tok;
			char next = lookAhead(0);
			if (next == ch) {
				/*this.currentMatch = new Token();
				this.currentMatch.tokenValue =
					String.valueOf(ch);
				this.currentMatch.tokenType = tok;*/
				consume(1);
			} else
				throw new ParseException(
					buffer + "\nExpecting  >>>" + ch + "<<< got >>>" 
					+ next + "<<<", ptr);
		}
		return this.currentMatch;
	}

	public void SPorHT() {
		try {
            char c = lookAhead(0);
            while (c == ' ' || c == '\t') {
				consume(1);
                c = lookAhead(0);
            }
        } catch (ParseException ex) {
			// Ignore
		}
	}
	
	/**
	 * JvB: utility function added to validate tokens
	 * 
	 * @see RFC3261 section 25.1:
	 * token       =  1*(alphanum / "-" / "." / "!" / "%" / "*"
                     / "_" / "+" / "`" / "'" / "~" )
     
     * @param c - character to check
	 * @return true iff character c is a valid token character as per RFC3261
	 */
	public static final boolean isTokenChar( char c ) {
		if ( isAlphaDigit(c) ) return true;
		else switch (c)
		{
			case '-':
	        case '.':
	        case '!':
	        case '%':
	        case '*':
	        case '_':
	        case '+':
	        case '`':
	        case '\'':
	        case '~':
	            return true;
	        default:
	            return false;
		}
	}
	
	
	public boolean startsId() {
		try {
			char nextChar = lookAhead(0);
            return isTokenChar(nextChar);
        } catch (ParseException ex) {
			return false;
		}
	}

	public boolean startsSafeToken() {
		try {
			char nextChar = lookAhead(0);
            if (isAlphaDigit(nextChar)) {
                return true;
            }
            else {
                switch (nextChar) {
                    case '_':
                    case '+':
                    case '-':
                    case '!':
                    case '`':
                    case '\'':
                    case '.':
                    case '/':
                    case '}':
                    case '{':
                    case ']':
                    case '[':
                    case '^':
                    case '|':
                    case '~':
                    case '%': // bug fix by Bruno Konik, JvB copied here
                    case '#':
                    case '@':
                    case '$':
                    case ':':
                    case ';':
                    case '?':
                    case '\"':
                    case '*':
                    case '=': // Issue 155 on java.net
                        return true;
                    default:
                        return false;
                }
            }
		} catch (ParseException ex) {
			return false;
		}
	}

	public String ttoken() {
		int startIdx = ptr;
		try {
			while (hasMoreChars()) {
				char nextChar = lookAhead(0);
                if ( isTokenChar(nextChar) ) {
                    consume(1);
                } else {
                	break;
                }
            }
			return buffer.substring(startIdx, ptr);
		} catch (ParseException ex) {
			return null;
		}
	}

	/* JvB: unreferenced
	public String ttokenAllowSpace() {
		int startIdx = ptr;
		try {
			while (hasMoreChars()) {
				char nextChar = lookAhead(0);
                if (isAlphaDigit(nextChar)) {
                    consume(1);
                }
                else {
                    boolean isValidChar = false;
                    switch (nextChar) {
                        case '_':
                        case '+':
                        case '-':
                        case '!':
                        case '`':
                        case '\'':
                        case '~':
                        case '%': // bug fix by Bruno Konik, JvB copied here
                        case '.':
                        case ' ':
                        case '\t':
                        case '*':
                            isValidChar = true;
                    }
                    if (isValidChar) {
                        consume(1);
                    }
                    else {
                        break;
                    }
                }

            }
			return buffer.substring(startIdx, ptr);
		} catch (ParseException ex) {
			return null;
		}
	}*/

	public String ttokenSafe() {
		int startIdx = ptr;
		try {
			while (hasMoreChars()) {
				char nextChar = lookAhead(0);
                if (isAlphaDigit(nextChar)) {
                    consume(1);
                }
                else {
                    boolean isValidChar = false;
                    switch (nextChar) {
                        case '_':
                        case '+':
                        case '-':
                        case '!':
                        case '`':
                        case '\'':
                        case '.':
                        case '/':
                        case '}':
                        case '{':
                        case ']':
                        case '[':
                        case '^':
                        case '|':
                        case '~':
                        case '%': // bug fix by Bruno Konik, JvB copied here
                        case '#':
                        case '@':
                        case '$':
                        case ':':
                        case ';':
                        case '?':
                        case '\"':
                        case '*':
                            isValidChar = true;
                    }
                    if (isValidChar) {
                        consume(1);
                    }
                    else {
                        break;
                    }
                }
            }
			return buffer.substring(startIdx, ptr); 
		} catch (ParseException ex) {
			return null;
		}
	}
    
    static final char ALPHA_VALID_CHARS = Character.MAX_VALUE;
    static final char DIGIT_VALID_CHARS = Character.MAX_VALUE - 1;
    static final char ALPHADIGIT_VALID_CHARS = Character.MAX_VALUE - 2;
    public void consumeValidChars(char[] validChars) {
        int validCharsLength = validChars.length;
		try {
			while (hasMoreChars()) {
				char nextChar = lookAhead(0);
                boolean isValid = false;
                for (int i = 0; i < validCharsLength; i++) {
                    char validChar = validChars[i];
                    switch(validChar) {
                        case ALPHA_VALID_CHARS:
                            isValid = isAlpha(nextChar);
                            break;
                        case DIGIT_VALID_CHARS:
                            isValid = isDigit(nextChar);
                            break;
                        case ALPHADIGIT_VALID_CHARS:
                            isValid = isAlphaDigit(nextChar);
                            break;
                        default:
                            isValid = nextChar == validChar;
                    }
                    if (isValid) {
                        break;
                    }
                }
                if (isValid) {
                    consume(1);
                }
                else {
                    break;
                }
            }
		} catch (ParseException ex) {
			
		}
    }

    /** Parse a comment string cursor is at a ". Leave cursor at closing "
	*@return the substring containing the quoted string excluding the
	* closing quote.
	*/
	public String quotedString() throws ParseException {
		int startIdx = ptr + 1;
		if (lookAhead(0) != '\"')
			return null;
		consume(1);
		while (true) {
			char next = getNextChar();
			if (next == '\"') {
				// Got to the terminating quote.
				break;
			} else if (next == '\0') {
				throw new ParseException(
					this.buffer + " :unexpected EOL",
					this.ptr);
			} else if (next == '\\') {
				consume(1);
			}
		}
		return buffer.substring(startIdx, ptr - 1);
	}

    /** Parse a comment string cursor is at a "(". Leave cursor at )
	*@return the substring containing the comment excluding the
	* closing brace.
	*/
	public String comment() throws ParseException {
		StringBuffer retval = new StringBuffer();
		if (lookAhead(0) != '(')
			return null;
		consume(1);
		while (true) {
			char next = getNextChar();
			if (next == ')') {
				break;
			} else if (next == '\0') {
				throw new ParseException(
					this.buffer + " :unexpected EOL",
					this.ptr);
			} else if (next == '\\') {
				retval.append(next);
				next = getNextChar();
				if (next == '\0')
					throw new ParseException(
						this.buffer + " : unexpected EOL",
						this.ptr);
				retval.append(next);
			} else {
				retval.append(next);
			}
		}
		return retval.toString();
	}

	/** Return a substring containing no semicolons.
	*@return a substring containing no semicolons.
	*/
	public String byteStringNoSemicolon() {
		StringBuffer retval = new StringBuffer();
		try {
			while (true) {
				char next = lookAhead(0);
				// bug fix from Ben Evans.
				if (next == '\0' || next == '\n' || next == ';' || next == ',' ) {
					break;
				} else {
					consume(1);
					retval.append(next);
				}
			}
		} catch (ParseException ex) {
			return retval.toString();
		}
		return retval.toString();
	}
	
	/**
	 * Scan until you see a slash or an EOL.
	 * 
	 * @return substring containing no slash.
	 */
	public String byteStringNoSlash() {
		StringBuffer retval = new StringBuffer();
		try {
			while (true) {
				char next = lookAhead(0);
				// bug fix from Ben Evans.
				if (next == '\0' || next == '\n' || next == '/'  ) {
					break;
				} else {
					consume(1);
					retval.append(next);
				}
			}
		} catch (ParseException ex) {
			return retval.toString();
		}
		return retval.toString();
	}

	/** Return a substring containing no commas
	*@return a substring containing no commas.
	*/

	public String byteStringNoComma() {
		StringBuffer retval = new StringBuffer();
		try {
			while (true) {
				char next = lookAhead(0);
				if (next == '\n' || next == ',') {
					break;
				} else {
					consume(1);
					retval.append(next);
				}
			}
		} catch (ParseException ex) {
		}
		return retval.toString();
	}

	public static String charAsString(char ch) {
		return String.valueOf(ch);
	}

	/** Lookahead in the inputBuffer for n chars and return as a string.
	 * Do not consume the input.
	 */
	public String charAsString(int nchars) {
        return buffer.substring(ptr, ptr + nchars);
	}

	/** Get and consume the next number.
	 *@return a substring corresponding to a number 
	 *(i.e. sequence of digits).
	 */
	public String number() throws ParseException {

		int startIdx = ptr;
		try {
			if (!isDigit(lookAhead(0))) {
				throw new ParseException(
					buffer + ": Unexpected token at " + lookAhead(0),
					ptr);
			}
			consume(1);
			while (true) {
				char next = lookAhead(0);
				if (isDigit(next)) {
					consume(1);
				} else
					break;
			}
			return buffer.substring(startIdx, ptr);
		} catch (ParseException ex) {
			return buffer.substring(startIdx, ptr);
		}
	}

	/** Mark the position for backtracking.
	 *@return the current location of the pointer.
	 */
	public int markInputPosition() {
		return ptr;
	}

	/** Rewind the input ptr to the marked position.
	 *@param position - the position to rewind the parser to.
	 */
	public void rewindInputPosition(int position) {
		this.ptr = position;
	}

	/** Get the rest of the String
	 * @return rest of the buffer.
	 */
	public String getRest() {
		if (ptr >= buffer.length())
			return null;
		else
			return buffer.substring(ptr);
	}

	/** Get the sub-String until the character is encountered
	 * @param c the character to match
	 * @return the substring that matches.
	 */
	public String getString(char c) throws ParseException {
		StringBuffer retval = new StringBuffer();
		while (true) {
			char next = lookAhead(0);
			//System.out.println(" next = [" + next + ']' + "ptr = " + ptr);
			//System.out.println(next == '\0');

			if (next == '\0') {
				throw new ParseException(
					this.buffer + "unexpected EOL",
					this.ptr);
			} else if (next == c) {
				consume(1);
				break;
			} else if (next == '\\') {
				consume(1);
				char nextchar = lookAhead(0);
				if (nextchar == '\0') {
					throw new ParseException(
						this.buffer + "unexpected EOL",
						this.ptr);
				} else {
					consume(1);
					retval.append(nextchar);
				}
			} else {
				consume(1);
				retval.append(next);
			}
		}
		return retval.toString();
	}

	/** Get the read pointer.
	 */
	public int getPtr() {
		return this.ptr;
	}

	/** Get the buffer.
	 */
	public String getBuffer() {
		return this.buffer;
	}

	/** Create a parse exception. 
	 */
	public ParseException createParseException() {
		return new ParseException(this.buffer, this.ptr);
	}
}
